package com.xiaotu.spider.project.pageprocessor;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.apache.commons.lang.StringEscapeUtils;
import org.apache.commons.lang.StringUtils;
import org.dom4j.Document;
import org.dom4j.Node;
import org.springframework.stereotype.Component;

import com.xiaotu.common.exception.SpiderException;
import com.xiaotu.common.exception.SpiderExceptionCode;
import com.xiaotu.common.util.Constants;
import com.xiaotu.common.util.DateUtils;
import com.xiaotu.common.util.FileUtil;
import com.xiaotu.common.util.PageUtils;
import com.xiaotu.common.util.RegexUtils;
import com.xiaotu.common.util.SepratorUtil;
import com.xiaotu.common.util.XMLUtils;
import com.xiaotu.spider.APageProcessor;

import net.sf.json.JSONObject;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Request;
import us.codecraft.webmagic.selector.Html;

/**
 * @类名 JZLYPageProcessor
 * @日期 2017年3月31日
 * @作者 高海军
 * @功能 九州龙渊数据爬取
 */
@Component("MBHuaTiPageProcessor")
public class MBHuaTiPageProcessor extends APageProcessor
{
	private static final String[] LIST_ARR = new String[] { "user", "time",
			"content" };
	
	private static final long startTime = DateUtils
			.getDate("2016-01-01 00:00", DateUtils.MINUTE_FORMAT).getTime();
	
	private static final long endTime = DateUtils
			.getDate("2017-04-01 00:00", DateUtils.MINUTE_FORMAT).getTime();
	
	private static boolean CAN_STOP = false;
	
	private static final String CSV_SEP = "~";
	
	private boolean canStop(String date)
	{
		if (CAN_STOP)
			return true;
		
		synchronized (MBHuaTiPageProcessor.class)
		{
			if (CAN_STOP)
				return true;
			else
			{
				long time = DateUtils.getDate(date, DateUtils.MINUTE_FORMAT)
						.getTime();
				if (time >= startTime)
					return false;
				else
				{
					CAN_STOP = true;
					return true;
				}
			}
		}
	}
	
	private boolean isRightDate(String date)
	{
		long time = DateUtils.getDate(date, DateUtils.MINUTE_FORMAT).getTime();
		if (time >= startTime && time < endTime)
			return true;
		return false;
	}
	
	@Override
	public void process(Page page)
	{
		try
		{
//			if (page.getUrl().toString().contains("http://weibo.com/tangque"))
			if (page.getUrl().toString().contains("http://weibo.com/jiuzhoutiankongcheng"))
				this.pageProcess(page);
			else
				this.jsonProcess(page);
		}
		catch (Exception e)
		{
			e.printStackTrace();
			FileUtil.writeFile(
					"D://" + RegexUtils.regexSearch("[0-9]+",
							page.getUrl().toString()) + "_"
							+ e.getClass().getSimpleName() + ".txt",
					StringEscapeUtils
							.unescapeJavaScript(page.getHtml().toString()));
			FileUtil.writeFile("D://err.csv",
					page.getUrl().toString() + SepratorUtil.LINE_SEPRATOR);
		}
	}
	
	/**
	 * 将数据集转成csv中的一行
	 * @param dataList 数据集
	 * @return csv行字符串
	 */
	public String list2CSVLine(List<Object> dataList)
	{
		if (dataList == null || dataList.isEmpty())
			return StringUtils.EMPTY;
		StringBuffer buffer = new StringBuffer();
		String text;
		for (Object data : dataList)
		{
			text = data == null ? null : data.toString();
			if (StringUtils.isEmpty(text))
				buffer.append(CSV_SEP);
			else
				buffer.append(
						CSV_SEP + "\"" + text.replaceAll("\"", "\"\"") + "\"");
		}
		return buffer.toString().substring(CSV_SEP.length());
	}
	
	private void jsonProcess(Page page)
	{
		
		String html = StringEscapeUtils.unescapeHtml(
				JSONObject.fromObject(page.getRawText()).getString("data"));
		
		if (StringUtils.isEmpty(html))
			throw new SpiderException(
					SpiderExceptionCode.DataInvalideException);
		
		Html htmlObj = new Html(html);
		
		List<String> divList = htmlObj
				.xpath("//div[@class='WB_cardwrap WB_feed_type S_bg2 ']").all();
		
		this.writeLine(divList, false);
		
	}
	
	private boolean writeLine(List<String> divList, boolean isPage)
	{
		for (String divStr : divList)
		{
			Html oneHtml = new Html(divStr);
			String user = oneHtml.xpath("//div[@class=\"face\"]/a/@title")
					.get();
			if (StringUtils.isEmpty(user))
				continue;
			
			String time = oneHtml
					.xpath("//div[@class='WB_from S_txt2']/a/@title").get();
			if (isPage)
			{
				if (this.canStop(time))
					return true;
			}
			
			if (!this.isRightDate(time))
				continue;
			
			String content = oneHtml.xpath("//div[@class='WB_text']/text()")
					.toString();
			List<String> valueList = oneHtml
					.xpath("//span[@class='line S_line1']//em[2]/text()").all();
			String up = valueList.get(valueList.size() - 1);
			if (!RegexUtils.regexMatch(RegexUtils.REGEX_INTEGER, up))
				up = "";
			String comment = valueList.get(valueList.size() - 2);
			if (!RegexUtils.regexMatch(RegexUtils.REGEX_INTEGER, comment))
				comment = "";
			
			String send = valueList.get(valueList.size() - 3);
			if (!RegexUtils.regexMatch(RegexUtils.REGEX_INTEGER, send))
				send = "";
			
			Object[] arr = new String[] { user.trim(), time.trim(),
					content.trim(), send.trim(), comment.trim(), up.trim() };
			
			String line = list2CSVLine(Arrays.asList(arr));
			System.out.println(line);
			FileUtil.writeFile("D:\\micblog.csv",
					line + SepratorUtil.LINE_SEPRATOR);
		}
		
		return false;
	}
	
	private void pageProcess(Page page)
	{
		
		String html = StringEscapeUtils
				.unescapeJavaScript(page.getHtml().toString());
		
		if (StringUtils.isEmpty(html))
			throw new SpiderException(
					SpiderExceptionCode.DataInvalideException);
		
		int start = html
				.indexOf("<div class=\"WB_feed WB_feed_v3 WB_feed_v4\"");
		int end = html.substring(start).indexOf("\"})</script>");
		String div = html.substring(start, start + end);
		
		Html divObj = new Html(div);
		
		List<String> divList = divObj
				.xpath("//div[@class='WB_cardwrap WB_feed_type S_bg2 ']").all();
		
		if (this.writeLine(divList, true))
			return;
		
		String pageNo = "1";
		if (page.getUrl().toString().contains("&page="))
		{
			pageNo = RegexUtils.regexSearch("&page=([0-9]+)",
					page.getUrl().toString());
			pageNo = RegexUtils.regexSearch("[0-9]+", pageNo);
		}
		
		// PageUtils.addTargetRequest(page,
		// "http://weibo.com/p/aj/v6/mblog/mbloglist?ajwvr=6&domain=103505&profile_ftype=1&is_all=1&pagebar=0&pl_name=Pl_Official_MyProfileFeed__22&id=1035051811696373&script_uri=/tangque&feed_type=0&page="
		// + pageNo + "&pre_page=" + pageNo
		// + "&domain_op=103505&__rnd=1491026873574",
		// page.getRequest().getExtras());
		// PageUtils.addTargetRequest(page,
		// "http://weibo.com/p/aj/v6/mblog/mbloglist?ajwvr=6&domain=103505&profile_ftype=1&is_all=1&pagebar=1&pl_name=Pl_Official_MyProfileFeed__22&id=1035051811696373&script_uri=/tangque&feed_type=0&page="
		// + pageNo + "&pre_page=" + pageNo
		// + "&domain_op=103505&__rnd=1491026873574",
		// page.getRequest().getExtras());
		
		PageUtils.addTargetRequest(page,
				"http://weibo.com/p/aj/v6/mblog/mbloglist?ajwvr=6&domain=100206&profile_ftype=1&is_all=1&pagebar=0&pl_name=Pl_Official_MyProfileFeed__27&id=1002065708949092&script_uri=/jiuzhoutiankongcheng&feed_type=0&page="
						+ pageNo + "&pre_page=" + pageNo
						+ "&domain_op=100206&__rnd=1491039925769",
				page.getRequest().getExtras());
		PageUtils.addTargetRequest(page,
				"http://weibo.com/p/aj/v6/mblog/mbloglist?ajwvr=6&domain=100206&profile_ftype=1&is_all=1&pagebar=1&pl_name=Pl_Official_MyProfileFeed__27&id=1002065708949092&script_uri=/jiuzhoutiankongcheng&feed_type=0&page="
						+ pageNo + "&pre_page=" + pageNo
						+ "&domain_op=100206&__rnd=1491039925769",
				page.getRequest().getExtras());
		
	}
	
	private String getContent(String xml)
	{
		try
		{
			Document document = XMLUtils.parseXML(xml);
			Node node = document.selectSingleNode("//div[@class='WB_text']");
			
			return "";
		}
		catch (Exception e)
		{
			throw new SpiderException(SpiderExceptionCode.DataInvalideException,
					e.getMessage());
		}
		
	}
	
	/**
	 * 设置要爬取的站点地址
	 */
	@SuppressWarnings({ "deprecation" })
	@Override
	protected void setSiteRequest()
	{
		Map<String, Object> tvMap = new HashMap<String, Object>();
		tvMap.put("tvid", "1");
		tvMap.put("tvName", "1");
		
		List<String> urlList = new ArrayList<String>();
		// urlList.add("http://weibo.com/tangque?profile_ftype=1&is_all=1#_0");
		urlList.add(
				"http://weibo.com/jiuzhoutiankongcheng?profile_ftype=1&is_all=1#_0");
		for (int i = 2; i < 50; i++)
			// urlList.add(
			// "http://weibo.com/tangque?is_search=0&visible=0&is_all=1&is_tag=0&profile_ftype=1&page="
			// + i + "#feedtop");
			urlList.add(
					"http://weibo.com/jiuzhoutiankongcheng?is_search=0&visible=0&is_all=1&is_tag=0&profile_ftype=1&page="
							+ i + "#feedtop");
		
		for (String url : urlList)
		{
			Request request = new Request(url);
			request.setExtras(tvMap);
			this.site.addStartRequest(request);
		}
		
	}
	
	@Override
	protected String getSpiderType()
	{
		return "JZLY";
	}
	
	@Override
	public String getPageDataType(Page page)
	{
		return Constants.PageDataType.WEIBO;
	}
	
	/*
	 * (non-Javadoc)
	 * @see
	 * com.xiaotu.spider.APageProcessor#getStartRequest(net.sf.json.JSONObject)
	 */
	@Override
	protected Request getStartRequest(JSONObject target)
	{
		return null;
	}
}
